import pandas as pd
import json
import re
from collections import defaultdict


# --------------------------
# 通用工具函数
# --------------------------
def parse_salary(salary_str):
    """统一薪资解析函数（支持范围值和单值）- 唯一正确版本"""
    if pd.isna(salary_str):
        return None

    salary_str = salary_str.strip().lower().replace("k", "")

    # 处理"xx以上"格式
    if "以上" in salary_str:
        min_val = float(salary_str.split("以上")[0])
        return {"type": "range", "value": [min_val, min_val]}

    # 处理范围格式（修正分隔符问题）
    parts = re.split(r"[-—~]", salary_str)
    if len(parts) == 2:
        return {"type": "range", "value": [float(parts[0]), float(parts[1])]}
    elif len(parts) == 1:
        try:
            return {"type": "single", "value": float(parts[0])}
        except ValueError:
            print(f"无效薪资格式：{salary_str}")
            return None

    return None

def clean_industry_field(industry_str):
    """行业字段标准化"""
    if pd.isna(industry_str):
        return "其他"

    replacements = {
        "互联网/电子商务": "互联网",
        "IT服务": "信息技术",
        "金融/银行/投资": "金融",
        "教育培训": "教育",
        "电子商务": "互联网"
    }

    for old, new in replacements.items():
        industry_str = industry_str.replace(old, new)

    return industry_str.split(" ")[0].strip()


def extract_tags(tag_str):
    """标签字段处理"""
    if pd.isna(tag_str):
        return []

    # 统一分隔符处理
    cleaned = re.sub(r"[^\u4e00-\u9fa5a-zA-Z0-9|]", ",", tag_str)
    tags = set()

    for t in re.split(r"[,，|]", cleaned):
        t = t.strip()
        if t:
            if "|" in t:
                tags.update(t.split("|"))
            else:
                tags.add(t)

    return list(tags)


def classify_city_group(city):
    """城市群分类"""
    yangtze_delta = ["上海", "杭州", "南京", "苏州", "无锡", "宁波"]
    pearl_delta = ["深圳", "广州", "珠海", "东莞", "佛山"]
    return "长三角" if city in yangtze_delta else "珠三角" if city in pearl_delta else "其他"


# --------------------------
# 数据预处理管道
# --------------------------
def load_and_clean_data():
    """数据加载与基础清洗"""
    df = pd.read_csv("拉勾网2023招聘数据.csv")

    # 统一缺失值处理
    df = df.dropna(subset=["city", "positionName", "salary"])

    # 添加衍生字段
    df["industry"] = df["industryField"].apply(clean_industry_field)
    df["tags"] = (df["positionLables"].fillna("") + "," + df["skillLables"].fillna("")).apply(extract_tags)
    df["city_group"] = df["city"].apply(classify_city_group)
    df["is_management"] = df["positionName"].str.contains(r"经理|总监|主管|总裁")

    # 薪资解析
    salary_data = df["salary"].apply(parse_salary)
    df["salary_min"] = salary_data.apply(
        lambda x: x["value"][0] if x and x["type"] == "range" else x["value"] if x else None)
    df["salary_max"] = salary_data.apply(
        lambda x: x["value"][1] if x and x["type"] == "range" else x["value"] if x else None)
    df["salary_avg"] = (df["salary_min"] + df["salary_max"]) / 2

    # 工作年限处理
    df["workYear"] = df["workYear"].str.extract(r"(\d+)").astype(float)
    df = df.dropna(subset=["workYear"])

    return df


# --------------------------
# 分析模块
# --------------------------
def analyze_city_salary(df):
    """城市平均薪资分析"""
    city_stats = df.groupby("city").agg(
        avg_salary=("salary_avg", "mean"),
        count=("salary_avg", "count")
    ).reset_index()

    # 过滤样本量过少的城市
    return city_stats[city_stats["count"] >= 10].set_index("city")["avg_salary"].to_dict()


def analyze_industry_distribution(df):
    """行业分布分析"""
    industry_counts = df["industry"].value_counts().reset_index()
    industry_counts.columns = ["industry", "count"]
    total = industry_counts["count"].sum()

    # 合并小类
    industry_counts["percentage"] = industry_counts["count"] / total
    industry_counts["industry"] = industry_counts.apply(
        lambda x: x["industry"] if x["percentage"] >= 0.02 else "其他", axis=1
    )

    return industry_counts.groupby("industry")["count"].sum().reset_index().to_dict("records")

def analyze_position_salaries(df):
    """岗位薪资分布分析（增强版）"""
    results = []

    for (industry, city, position), group in df.groupby(["industry", "city", "positionName"]):
        try:
            # 数据有效性检查
            if group[["salary_min", "salary_max"]].isnull().any().any():
                continue

            salaries = []
            for _, row in group.iterrows():
                salaries.extend([row["salary_min"], row["salary_max"]])

            if len(salaries) < 20:
                continue

            salary_series = pd.Series(salaries)
            Q1 = salary_series.quantile(0.25)
            Q3 = salary_series.quantile(0.75)
            IQR = Q3 - Q1

            filtered = salary_series[
                (salary_series >= (Q1 - 1.5 * IQR)) &
                (salary_series <= (Q3 + 1.5 * IQR))
                ]

            if len(filtered) < 10:
                continue

            # 城市分类保护
            try:
                city_group = classify_city_group(city)
            except:
                city_group = "其他"

            # 构建结果
            result = {
                "industry": str(industry),
                "city": str(city),
                "position": str(position),
                "city_group": city_group,
                "min": round(float(filtered.min()), 1),
                "q1": round(float(Q1), 1),
                "median": round(float(filtered.median()), 1),
                "q3": round(float(Q3), 1),
                "max": round(float(filtered.max()), 1),
                "sample_size": int(len(filtered)),
                "outliers": int(len(salary_series) - len(filtered))
            }

            # 关键字段验证
            if any(pd.isna(result[k]) for k in ["min", "q1", "median"]):
                continue

            results.append(result)

        except Exception as e:
            print(f"Error processing {position}: {str(e)}")
            continue

    return results


def analyze_education_promotion(df):
    """教育晋升分析"""
    result = {"city_groups": {}, "max_workYear": int(df["workYear"].max())}

    for city_group in ["长三角", "珠三角"]:
        group_df = df[df["city_group"] == city_group]
        result["city_groups"][city_group] = {"management": {}, "technical": {}}

        for position_type in ["management", "technical"]:
            type_df = group_df[group_df["is_management"] == (position_type == "management")]

            for edu in ["硕士", "本科"]:
                edu_df = type_df[type_df["education"] == edu]
                if not edu_df.empty:
                    stats = edu_df.groupby("workYear")["salary_avg"].agg(["mean", "count"])
                    result["city_groups"][city_group][position_type][edu] = {
                        str(year): data["mean"].round(1)
                        for year, data in stats.iterrows()
                        if data["count"] >= 5
                    }

    return result


def analyze_tag_frequency(df):
    """标签频率分析"""
    tag_counts = defaultdict(int)
    for tags in df["tags"]:
        for tag in tags:
            tag_counts[tag] += 1
    return [{"name": k, "value": v} for k, v in tag_counts.items()]


# --------------------------
# 主处理流程
# --------------------------
def main_processing():
    # 数据加载与清洗
    df = load_and_clean_data()

    # 执行各项分析
    analyses = {
        "city_salary": analyze_city_salary(df),
        "industry_distribution": analyze_industry_distribution(df),
        "position_salaries": analyze_position_salaries(df),
        "education_promotion": analyze_education_promotion(df),
        "tag_frequency": analyze_tag_frequency(df)
    }

    # 保存所有结果
    for name, data in analyses.items():
        with open(f"{name}.json", "w", encoding="utf-8") as f:
            json.dump(data, f, ensure_ascii=False, indent=2)


if __name__ == "__main__":
    main_processing()